#!/usr/bin/python3

# Copyright (c) 2011 Michael Mol <mikemol@gmail.com>.
#
# This work is licensed under a
# Creative Commons Attribution 3.0 Unported License, viewable at
# http://creativecommons.org/licenses/by/3.0/
#
import symbolisms
import filterstack
import chortfilters

inchain = filterstack.chainBuild(chortfilters.url_filter_in)
filterstack.chainAppend(inchain, chortfilters.attention_filter_in)
filterstack.chainAppend(inchain, chortfilters.case_tolower_in)
filterstack.chainAppend(inchain, chortfilters.de_stretch_speeeech_in)
filterstack.chainAppend(inchain, chortfilters.numbers_reduce_in)
filterstack.chainAppend(inchain, chortfilters.punc_in)

outchain = filterstack.chainBuild(chortfilters.punc_out)
filterstack.chainAppend(outchain, chortfilters.numbers_reduce_out)
filterstack.chainAppend(outchain, chortfilters.url_filter_out)


def core():
    symbolisms.init(inchain, outchain)

    lineCount = 0
    processedLineCount = 0
    roundCount = 0
    while True:
        line = None
        try:
            lineCount += 1
            line = input()
        except EOFError:
            pass
        except UnicodeDecodeError:
            # Bad input line
            # Skip it.
            print( "Skipping line %d; bad Unicode data" % lineCount)
            continue
        
        if line == None:
            break
        tokens = line.split()
        symbolisms.process(tokens)
        processedLineCount += 1
    
    #    # Really useful for watching the model output change as you dump data into it.
    #    if lineCount == 1000:
    #        lineCount = 0
    #        roundCount += 1
    #        seq = symbolisms.genSequence()
    ##        print '((',roundCount,'))',
    #        for sym in seq:
    #            print sym,
    #        print
        if 0 == (processedLineCount % 10000):
            symbolisms.save()
            print("Committed %d" % processedLineCount)
    
    symbolisms.save()
#    # Let's see what we've got for output.
#    for x in range(1):
#        seq = symbolisms.genSequence()
#        for symb in seq:
#            print( symb, end=' ')
#        print()


import cProfile

cProfile.run('core()')
